#include "StdAfx.h"
#include "BasicFunction.h"
#include <complex>
#include "shlwapi.h"
#include "afxdisp.h"
#include "math.h"
#include "IO.h"
#define MAXWORDS (int)10000000
#define MAXLINE 10000
CBasicFunction::CBasicFunction(void)
{
}

CBasicFunction::~CBasicFunction(void)
{
}

//string to file
bool CBasicFunction::stringToFile(CString str, CString caption)
{
	//define a dialog to open a file
	CFile cfile;
	CString strFilter;
	CFileException e;
	strFilter = "text file (*.txt)|*.txt||";
	//display the dialog
	CFileDialog dlg(FALSE,NULL,NULL,OFN_HIDEREADONLY | OFN_OVERWRITEPROMPT | OFN_EXPLORER,strFilter);
	dlg.m_ofn.lpstrTitle = caption;
	if(dlg.DoModal()!=IDOK)
	{
		return 0;
	}
	CString strFileName;
	CString strExtension;
	//return the name of the file
	strFileName = dlg.m_ofn.lpstrFile;
	//convert the CString to char*
	char* strChar = CBasicFunction::CStringToChar(str);
	//open the file and save the string
	if(!cfile.Open(strFileName, CFile::modeRead|CFile::modeCreate|CFile::modeWrite, &e))
	{
		::AfxMessageBox(CString("Failed to open the file!"), MB_OK, 0);
		return 0;
	}
	cfile.Write(strChar, str.GetLength());
	cfile.Close();

	free(strChar);
	return 1;
}

//write the string to log file
bool CBasicFunction::WriteToLog(CString str)
{
//define char pointer for file name and folder name;
	CFile cfile;
	CFileException e;
	CString logfile("C:\\VCLog.txt");
//get the system time for the message
	LPSYSTEMTIME lptime = new SYSTEMTIME;
	::GetSystemTime(lptime);
	CString logmeg(" //Time: ");
	logmeg += CBasicFunction::douTostr(lptime->wYear);
	logmeg += " ";
	logmeg += CBasicFunction::douTostr(lptime->wMonth);
	logmeg += " ";
	logmeg += CBasicFunction::douTostr(lptime->wDay);
	logmeg += " ";
	logmeg += CBasicFunction::douTostr(lptime->wHour);
	logmeg += ":";
	logmeg += CBasicFunction::douTostr(lptime->wMinute);
	logmeg += ":";
	logmeg += CBasicFunction::douTostr(lptime->wMinute);
	logmeg += "\r\n";
	str += logmeg;
	CStringA stringtofile(str);

//open the file to read
	if(!cfile.Open(logfile, CFile::modeRead|CFile::modeWrite, &e))
	{
		free(lptime);
		e.ReportError();
		return 0;
	}
//write to file
	try
	{
		cfile.SeekToEnd();
		cfile.Write(stringtofile, (UINT)stringtofile.GetLength());
	}
	catch(CException* e)
	{
		free(lptime);
		e->ReportError();
		return 0;
	}
	cfile.Close();
	free(lptime);
	return 1;
}

//double to string
CString CBasicFunction::douTostr( double dou )
{
	std::ostringstream systemstring;
	systemstring << dou;
	return CString(systemstring.str().c_str());
}

//calculate the Euclidean distance between two double array
double CBasicFunction::eDistance(double* data1, double* data2, int NumElement)
{
	double m_distance = 0;
	for(int i=0; i<NumElement; i++)
	{
		double temp1 = data1[i]+0;
		double temp2 = data2[i]+0;
		m_distance += /*temp2*temp2;*/(temp1-temp2)*(temp1-temp2);
	}
	return sqrt(m_distance);
}

//transpose a matrix
double** CBasicFunction::matrixTransp(double** m_data, int m_column, int m_row)
{
	double** m_matrix = (double**)malloc(sizeof(double*)*m_column);
	for(int i=0; i<m_column; i++)
	{
		m_matrix[i] = (double*)malloc(sizeof(double)*m_row);
		for(int j=0; j<m_row; j++)
		{
			m_matrix[i][j] = m_data[j][i];
		}
	}
	return m_matrix;
}

//normalize data
bool CBasicFunction::normalizedData(double** un_data, int un_c, int un_r, double* n_ave, double* n_std)
{
//exmaine
	if(un_r == 0 || un_c == 0 || un_c == 1)
	{
		::AfxMessageBox(CString("either coloum or row is equal to 0 or only 1 data"), MB_OK, 0);
		return 0;
	}
	double* un_sum = (double*)calloc(un_r, sizeof(double));
	for(int i=0; i<un_r; i++)
		for(int j=0; j<un_c; j++)
			un_sum[i] += un_data[i][j];

	for(int i=0; i<un_r; i++)
	{
		n_ave[i] = un_sum[i]/un_c;
		un_sum[i] = 0;
	}

	for(int i=0; i<un_r; i++)
		for(int j=0; j<un_c; j++)
			un_sum[i] += (un_data[i][j]-n_ave[i])*(un_data[i][j]-n_ave[i]);

	for(int i=0; i<un_r; i++)
		n_std[i] = sqrt(un_sum[i]/(un_c-1));

	for(int i=0; i<un_r; i++)
	{
		if(n_std[i]!=0)
			for(int j=0; j<un_c; j++)
				un_data[i][j] = (un_data[i][j]-n_ave[i])/n_std[i];
		else
			for(int j=0; j<un_c; j++)
				un_data[i][j] = 0;
	}
	free(un_sum);
	return 1;
}

//search forward for a target char
int CBasicFunction::SearchForward(char m_t, char* m_s, int m_cur, int m_len)
{
	int target = m_cur+1;
	bool flag = 0;
	while(target<m_len && !flag)
	{
		if(m_s[target]!=m_t)
			target ++;
		else
			flag = 1;
	}
	return target;
}

//Search forward for a target char
unsigned int CBasicFunction::SearchForward(char target, const char* sourcestring, unsigned int currentposition,
											unsigned int sourcelength)
{
	unsigned position = currentposition;
	while(position<sourcelength)
	{
		if(sourcestring[position]!=target)
			position ++;
		else
			break;
	}
	return (position==sourcelength)?((unsigned int)-1):position;
}

//search backward for a target char
int CBasicFunction::SearchBackward(char m_t, char* m_s, int m_cur, int m_len)
{
	int target = m_cur-1;
	bool flag = 0;
	while(target>=0 && !flag)
	{
		if(m_s[target]!=m_t)
			target --;
		else
			flag = 1;
	}
	return target;
}

//Search backward for a target char
unsigned int CBasicFunction::SearchBackward(char target, const char* sourcestring,
											unsigned int currentposition, unsigned int sourcelength)
{
	unsigned position = currentposition;
	while(position>0)
	{
		if(sourcestring[position]!=target)
			position --;
		else
			break;
	}

	if(position>0)
		return position;
	else
		return (sourcestring[position]==target)?position:((unsigned int)-1);
}

//search forward for a string
int CBasicFunction::SearchStr(char* m_t, char* m_s, int m_cur, int m_tlen, int m_slen)
{
	//validate the match
	if(m_tlen<=0)
		return -1;
	if(m_tlen>m_slen)
		return -1;

	//initialize
	int target = m_cur+1;
	bool flag = 0;
	//start matching from a position
	while((target+m_tlen)<=m_slen)
	{
		bool tempflag = 1;
		int i=0;
		//match the string from the position
		while(tempflag && i<m_tlen)
		{
			if((m_s[target+i]>='a' && m_s[target+i]<='z'))
			{
				if(m_s[target+i]!=m_t[i] && (m_s[target+i]-32)!=m_t[i])
				{
					tempflag = 0;
					break;
				}
			}
			else
			{
				if(m_s[target+i]>='A' && m_s[target+i]<='Z')
				{
					if(m_s[target+i]!=m_t[i] && (m_s[target+i]+32)!=m_t[i])
					{
						tempflag = 0;
						break;
					}
				}
				else
				{
					if(m_s[target+i]!=m_t[i])
					{
						tempflag = 0;
						break;
					}
				}
			}
			i++;
		}
		if(tempflag)
			break;
		else
			target ++;
	}

	return target;
}

//search forward for a string
unsigned int CBasicFunction::SearchStr(const char* m_t, const char* m_s, unsigned int m_cur,
									   unsigned int m_tlen, unsigned int m_slen)
{
	//validate the match
	if(m_tlen==0)
		return -1;
	if(m_tlen>m_slen)
		return -1;
	if(m_slen==0)
		return -1;

	//initialize
	unsigned int target = m_cur;
	bool flag = 0;
	//start matching from a position
	while((target+m_tlen)<m_slen)
	{
		bool tempflag = true;
		unsigned int i=0;
		//match the string from the position
		while(i<m_tlen)
		{
			if(m_s[target+i]!=m_t[i])
			{
				tempflag = false;
				break;
			}
			i++;
		}
		if(tempflag)
			break;
		else
			target ++;
	}
	
	if(target+m_tlen==m_slen)
		return -1;
	else
		return target;
}

//convert a CString to char*
char* CBasicFunction::CStringToChar(CString str)
{
	string newstring = CT2CA(str);
	char* target = new char[newstring.size()];
	for(unsigned int i=0; i<newstring.size(); i++)
		target[i] = newstring[i];
	return target;
}

//validate if the dot is a full stop or not
bool CBasicFunction::ValidateFullStop(int curpos, char* m_s, int m_slen)
{
	if(curpos>=m_slen-3)
	{
		//curpos is at the end, return fullstop
		return 1;
	}
	if(m_s[curpos+1]!=' '&&m_s[curpos+1]!='\t'&&m_s[curpos+1]!='\n'&&m_s[curpos+1]!='\r')
	{
		//next char is not empty, return non fullstop
		return 0;
	}
	if(m_s[curpos+1]=='\t'||m_s[curpos+1]=='\n'||m_s[curpos+1]=='\r')
	{
		//next char is tap, enter, or end of line, return fullstop
		return 1;
	}
	if((m_s[curpos+1]==' ')&&((m_s[curpos+2]>='A'&&m_s[curpos+2]<='Z')||m_s[curpos+2]==' '||
		m_s[curpos+2]=='\t'||m_s[curpos+2]=='\r'||m_s[curpos+2]=='\n'))
	{
		//if next char is ' ', and the one after next is captical, blank, tap, 
		//enter, or end of line, return fullstop
		return 1;
	}
	//else return non fullstop
	return 0;
}

//cubic weight function
double CBasicFunction::CubicWeight(double x)
{
	double rx = (x+(double)2<=0)?0:pow(x+(double)2, (double)3);
	rx += -(double)4*(x+(double)1<=0?0:pow(x+(double)1, (double)3));
	rx += 6*(x<=0?0:pow(x, (double)3));
	rx += -4*(x-(double)1<=0?0:pow(x-(double)1, (double)3));
	return rx/6;
}

//generate a matrix with random doubles
double** CBasicFunction::RandomDoubleMatrix(int m_c, int m_r)
{
	//validate the number of columsn and rows
	if(m_c<=0)
	{
		::AfxMessageBox(CString("Illegal Column"), MB_OK, 0);
		//return -1
		double** m_matrix = (double**)malloc(sizeof(double*)*1);
		m_matrix[0] = (double*)malloc(sizeof(double)*1);
		m_matrix[0][0] = -1;
		return m_matrix;
	}
	if(m_r<=0)
	{
		::AfxMessageBox(CString("Illegal Row"), MB_OK, 0);
		//return -1
		double** m_matrix = (double**)malloc(sizeof(double*)*1);
		m_matrix[0] = (double*)malloc(sizeof(double)*1);
		m_matrix[0][0] = -1;
		return m_matrix;
	}

	//seed time as random
	srand((unsigned)time(NULL));
	//create memory
	double** m_matrix = (double**)malloc(sizeof(double*)*m_r);
	//loop to generate two int for doubles
	for(int i=0; i<m_r; i++)
	{
		m_matrix[i] = (double*)malloc(sizeof(double)*m_c);
		for(int j=0; j<m_c; j++)
			m_matrix[i][j] = (double)rand()/max(0.00000000001,(double)rand());
	}
	//return the double**
	return m_matrix;
}

//convert an integer to char*
char* CBasicFunction::IntToChar(int m_source)
{
	std::ostringstream systemstring;
	systemstring << m_source;
	char* target = new char[systemstring.str().size()];
	for(unsigned int i=0; i<systemstring.str().size(); i++)
		target[i] = systemstring.str()[i];
	return target;
}

//conver a double vector to char*
char* CBasicFunction::DoubleVectorToChar(double* m_Double, int m_DoubleSize, int m_Precision, char m_Separator, int* m_CharSize)
{
	std::ostringstream systemstring;
	systemstring.precision(m_Precision);

	//Print the double to systemstring
	for(int i=0; i<m_DoubleSize; i++)
		systemstring << m_Double[i] << m_Separator;
	char* target = new char[systemstring.str().size()];
	for(unsigned int i=0; i<systemstring.str().size(); i++)
		target[i] = systemstring.str()[i];
	*m_CharSize = (int)systemstring.str().size();
	return target;
}

//convert a double matrix into a char*
char* CBasicFunction::DoubleMatrixToChar(double** m_Double, int m_DoubleX, int m_DoubleY, int m_Precision, char m_Separator, int* m_CharSize)
{
	std::ostringstream systemstring;
	systemstring.precision(m_Precision);

	//Print the double matrix to systemstring
	for(int i=0; i<m_DoubleX; i++)
	{
		for(int j=0; j<m_DoubleY; j++)
		{
			systemstring << m_Double[i][j] << m_Separator;
		}
		systemstring << "\r\n";
	}

	//Create the char*
	char* target = new char[systemstring.str().size()];
	for(unsigned int i=0; i<systemstring.str().size(); i++)
		target[i] = systemstring.str()[i];
	*m_CharSize = (int)(systemstring.str().size());

	return target;
}

//load doubles from a string char*
double* CBasicFunction::CharToDoubleVector(char* m_Source, int m_SourceLength, char m_Separator,
										   int* m_Converted)
{
	//get the number of doubles
	int i=0;
	*m_Converted = 0;
	while(i<m_SourceLength)
	{
		if(i<m_SourceLength-1)
			*m_Converted += 1;
		i = CBasicFunction::SearchForward(m_Separator, m_Source, i, m_SourceLength);
	}
	//locate the memory
	double* m_DoubleFromChar = (double*)malloc(sizeof(double)*(*m_Converted));
	//loop to load the doubles
	int m_Count = 0;
	i = 0;
	while(i<m_SourceLength)
	{
		if(i<m_SourceLength-1)
			m_DoubleFromChar[m_Count++] = atof(&m_Source[i]);
		i = CBasicFunction::SearchForward(m_Separator, m_Source, i, m_SourceLength)+1;
	}
	//validate the results
	if(m_Count != *m_Converted)
	{
		*m_Converted = 0;
		return m_DoubleFromChar;
	}
	//return the doubles
	return m_DoubleFromChar;
}

//Voting
int CBasicFunction::OnVoting(int *m_VotingData, int m_NumberVote, int m_DataLength)
{
	if(m_DataLength<=0)
	{
		WriteToLog(CString("Don't be silly, you have noting to vote"));
		return -1;
	}
	int* m_VoteSum = (int*)calloc(m_NumberVote, sizeof(int));

	//Calculate the number of each vote
	for(int i=0; i<m_DataLength; i++)
	{
		m_VoteSum[m_VotingData[i]] ++;
	}

	//Get the maximum vote
	int m_MaxVote = 0;
	int m_MaxVoteNumber = 0;
	for(int i=0; i<m_NumberVote; i++)
	{
		if(m_VoteSum[i]>m_MaxVote)
		{
			m_MaxVote = m_VoteSum[i];
			m_MaxVoteNumber = i;
		}
	}

	//Return the maximum vote
	free(m_VoteSum);
	return m_MaxVoteNumber;
}

//get the dimension of the input matrix
bool CBasicFunction::GetDataDimension(char* f_input, int f_len, int* d_column, int* d_row)
{
//create a temporal char* for storing the converted char*
	char d_convert[50];
//convert the number of columns
	//search for the beginning and ending of the converted chars
	int d_start=0;
	int d_end = CBasicFunction::SearchForward(';', f_input, d_start, f_len);
	//if legal end found, assgin the chars to d_convert for converting
	if(d_end<f_len)
	{
		for(int i=0; i<(d_end-d_start); i++)
			d_convert[i] = f_input[d_start+i];
		d_convert[d_end-d_start] = NULL;
	//convert the char
		*d_column = (int)atof(d_convert);
	}
	else
		return 0;
//convert the number of rows
	d_start = d_end+1;
	d_end = CBasicFunction::SearchForward(';', f_input, d_start, f_len);
	//legal end found, assign the chars for converting
	if(d_end<f_len)
	{
		for(int i=0; i<(d_end-d_start); i++)
			d_convert[i] = f_input[d_start+i];
		//inseart end after the last char
		d_convert[d_end-d_start] = NULL;
		//conver the char
		*d_row = (int)atof(d_convert);
	}
	else
		return 0;
	return 1;
}

//read the data into the matrix
bool CBasicFunction::ReadMatrixData(char* f_input, int f_len, int d_column, int d_row, double** m_data)
{
	//define a char for converting
	char d_convert[50];
	//search forward to the start of matrix data
	int d_start = CBasicFunction::SearchForward('\n', f_input, 0, f_len);
	d_start ++;
	//validate the location of d_start
	if(d_start>=f_len)
	{
		::AfxMessageBox(CString("File length not correct, Please check!"), MB_OK, 0);
		return 0;
	}
	//start to read data;
	int d_end;
	int count = 0;	//count the number of data read
	int cur_column = 0;
	int cur_row = 0;
	//loop to load the data until d_start reach the end
	while(d_start<f_len)
	{
		//search for the end of data
		d_end = CBasicFunction::SearchForward(';', f_input, d_start, f_len);
		//validate and load the chars to be converted
		if(d_end<f_len)
		{
			if(d_end-d_start<50)
			{
				for(int i=0; i<(d_end-d_start); i++)
					d_convert[i] = f_input[d_start+i];
				d_convert[d_end-d_start] = NULL;
				//convert the chars to double and increase the current position by 1
				m_data[cur_row][cur_column] = atof(d_convert);
				cur_column ++;
				//if current column reach the limit, restart from 0 and increase the row by 1
				if(cur_column >= d_column)
				{
					cur_column = 0;
					cur_row ++;
				}
				count ++;
			}
			else
			{
				CString error("Error!!\r\nPlease check data at location: (");
				error += CBasicFunction::douTostr(cur_row);
				error += CString(", ");
				error += CBasicFunction::douTostr(cur_column);
				error += CString(").");
				::AfxMessageBox(error, MB_OK, 0);
				return 0;
			}
		}	
		d_start = d_end+1;
	}
	//validate the number of data read
	if(count != d_column*d_row)
	{
		CString error("Error!!\r\nNumber of data indicate are: ");
		error += CBasicFunction::douTostr(d_column*d_row);
		error += CString("\r\nNumber of data read are: ");
		error += CBasicFunction::douTostr(count);
		::AfxMessageBox(error, MB_OK, 0);
		return 0;
	}
	return 1;
}

//Load double numbers from a file
double** CBasicFunction::LoadDoubleFromFile(CString FileName, int* Rows, int* Columns)
{
	//File fail to open the file, return a double matrix with only 1 element of -1.0
	CFile doublefile;
	try
	{
		doublefile.Open(FileName, CFile::modeRead);
	}
	catch(CException *e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		throw string(ascii);
	}
	//Read the file
	int numberchar = (int)doublefile.GetLength();
	char* filechar = new char[numberchar];
	UINT ByteRead = doublefile.Read(filechar, numberchar);
	doublefile.Close();
	//Get the data dimension
	CBasicFunction::GetDataDimension(filechar, numberchar, Columns, Rows);
	//Locate memory for the double matrix
	double** doubledata = new double*[*Rows];
	for(int i=0; i<*Rows; i++)
		doubledata[i] = new double[*Columns];
	//Read the data
	CBasicFunction::ReadMatrixData(filechar, numberchar, *Columns, *Rows, doubledata);
	return doubledata;
}

//read data from a file
vector<vector<double> > CBasicFunction::LoadDoubleFromFile(std::string filename, std::string separator)
{
	vector<vector<double> > datafromfile;
	string datastring("0123456789");
	std::ifstream datafile(filename.c_str(), std::ios_base::in);
	while(!datafile.eof())
	{
		//get the current line of data
		string currentline;
		std::getline(datafile, currentline);
		//load data into vector
		vector<double> currentvector;
		typedef string::size_type stringite;
		stringite pos=currentline.find_first_of(separator);
		stringite startposition = 0;
		//loop to search data and add to the currentvector
		while(pos!=string::npos)
		{
			string tempstring = string(&(currentline.c_str()[startposition]), pos-startposition);
			startposition=pos+1;
			pos = currentline.find_first_of(separator, startposition);
			if(tempstring.find_first_of(datastring)==string::npos)
				continue;
			currentvector.push_back(atof(tempstring.c_str()));
		}
		if(!currentvector.empty())
			datafromfile.push_back(currentvector);
	}
	datafile.close();
	return datafromfile;
}

//Append the content to an existing file
bool CBasicFunction::AppendToFile(CString m_filename, CString m_content)
{
	CFile filetowrite;
	CFileException e;
	//If fail to open the file
	if(!filetowrite.Open(m_filename, CFile::modeWrite|CFile::modeRead, &e))
	{
		//Try to create the file
		try
		{
			filetowrite.Open(m_filename, CFile::modeCreate|CFile::modeWrite);
		}
		//If still failed, report error and return false;
		catch(CException *ee)
		{
			ee->ReportError();
			return 0;
		}
	}
	//Seek to end
	filetowrite.SeekToEnd();
	CStringA contenttowrite(m_content);
	filetowrite.Write(contenttowrite.GetBuffer(), (UINT)contenttowrite.GetLength());
	filetowrite.Close();
	return 1;
}

//Delete a matrix
void CBasicFunction::DeleteMatrix(double **input, int row)
{
	for(int i=0; i<row; i++)
		free(input[i]);
	free(input);
	return;
}

//Write to a file, if already existed, overwrite it automatically
bool CBasicFunction::WriteToFile(CString m_filename, CString m_content)
{
	//Open the file
	CFile filetowrite;
	CFileException e;
	//If fail to open the file
	if(!filetowrite.Open(m_filename, CFile::modeWrite|CFile::modeRead|CFile::modeCreate, &e))
	{
		e.ReportError();
		return 0;
	}
	//Write m_content to the file
	CStringA contenttowrite(m_content);	//Convert the content into ascii string
	filetowrite.Write(contenttowrite.GetBuffer(), (UINT)contenttowrite.GetLength());
	filetowrite.Close();
	return 1;
}

//Rescale a matrix
void CBasicFunction::MatrixRescale(double** input, int row, int column, double newmin, double newmax)
{
	//Get the max and min value
	double maxvalue;
	double minvalue;
	CBasicFunction::MatrixMaxMin(input, row, column, &maxvalue, &minvalue);
	if(maxvalue == minvalue)
	{
		return;
	}

	//Rescale
	for(int i=0; i<row; i++)
	{
		for(int j=0; j<column; j++)
		{
			input[i][j] = (input[i][j]-minvalue)/(maxvalue-minvalue)*(newmax-newmin) + newmin;
		}
	}
	return;
}

//Get the max and min of a matrix
void CBasicFunction::MatrixMaxMin(double** input, int row, int column, double* maxvalue, double* minvalue)
{
	//temporal define the max and min
	double maxi = -(double)ULONG_MAX;
	double mini = (double)ULONG_MAX;
	for(int i=0; i<row; i++)
	{
		for(int j=0; j<column; j++)
		{
			if(input[i][j]>maxi)
				maxi = input[i][j];
			if(input[i][j]<mini)
				mini = input[i][j];
		}
	}
	//Assigne the value and return;
	*maxvalue = maxi;
	*minvalue = mini;
	return;
}

//Calculate the threshold of a histogram using Otsu (1979)'s method
double CBasicFunction::GetOtsuThreshold(double *histogram, int sizeofhistogram)
{
	double threshold = 0;
	double highest = 0;
	for (int i = 1 ; i < sizeofhistogram-1 ; i ++ )
	{
		double mean_0 = 0 ;
		double mean_1 = 0 ;
		double var_0 = 0 ;
		double var_1 = 0 ;
		for (int j = 0 ; j <= i ; j ++ )
		{
			mean_0 = mean_0 + histogram[j] ;
			var_0 = var_0 + histogram[j] * (double) j ;
		}
		for (int j = i + 1 ; j < sizeofhistogram ; j ++ )
		{
			mean_1 = mean_1 + histogram[j] ;
			var_1 = var_1 + histogram[j] * (double) j ;
		}
		var_0 = var_0 / mean_0 ;
		var_1 = var_1 / mean_1 ;
		double temp1 = mean_0 * mean_1 * ( var_0 - var_1 ) * ( var_0 - var_1 );
		if ( temp1 > highest )
		{
			threshold = i ;
			highest = temp1 ;
		}
	}
	return threshold;
}

//Get the max and min values of a vector
void CBasicFunction::VectorMaxMin(double *input, int inputsize, double *maxvalue, double *minvalue)
{
	*minvalue = (double)ULONG_MAX;
	*maxvalue = -(double)ULONG_MAX;
	//Loop to get the max and min
	for(int i=0; i<inputsize; i++)
	{
		if(*maxvalue<input[i])
			*maxvalue = input[i];
		if(*minvalue>input[i])
			*minvalue = input[i];
	}
	return;
}

//Rescale a vector
void CBasicFunction::VectorRescale(double *input, int inputsize, double maxvalue, double minvalue)
{
	double oldmax, oldmin;
	//Get the max and min of the input
	CBasicFunction::VectorMaxMin(input, inputsize, &oldmax, &oldmin);
	for(int i=0; i<inputsize; i++)
	{
		input[i] = (input[i]-oldmin)/(oldmax-oldmin)*(maxvalue-minvalue) + minvalue;
	}
}

//Interpolation of a matrix
double** CBasicFunction::MatrixInterpolation(double **input, int row, int column, int newrow, int newcolumn)
{
	//define parameters used during the loop
	double old_w;
	double old_h;
	int old_j;
	int old_i;
	double dy;
	double dx;
	double rx;
	double ry;

	//Create a new matrix
	double** newmatrix = (double**)malloc(sizeof(double*)*newrow);
	for(int i=0; i<newrow; i++)
	{
		newmatrix[i] = (double*)calloc(newcolumn, sizeof(double));
		for(int j=0; j<newcolumn; j++)
		{
			//determine its position in the old image
			old_w = (double)row/(double)newrow*(double)i;
			old_h = (double)column/(double)newcolumn*(double)j;
			old_j = (int)old_h;
			old_i = (int)old_w;
			dy = old_h - (double)old_j;
			dx = old_w - (double)old_i;
			double newdata = 0;
			//interpolase
			for(int k=-1; k<3; k++)
			{
				for(int l=-1; l<3; l++)
				{
					rx = CBasicFunction::CubicWeight(dy-k);
					ry = CBasicFunction::CubicWeight(l-dx);
					newdata += input[old_j][old_i]*rx*ry;
				}
			}
			newmatrix[i][j] = newdata;
		}
	}
	return newmatrix;
}

/********************************************
*											*
*		Mikhail's basic and misc			*
*				functions					*
*											*
********************************************/

std::string CBasicFunction::IntToString (int num) // Convert int to std::string
{
	std::ostringstream s;
	s << num;
	return s.str();
}

CString CBasicFunction::IntToCString (int num) // Convert int CString
{
	std::ostringstream s;
	s << num;
	std::string std_temp = s.str();
	CString strCString(std_temp.c_str());
	return strCString;
}

std::string CBasicFunction::BoolToString (bool thisBool) // Converts a bool to std::string
{
	return (thisBool)?"TRUE":"FALSE";
}

CString CBasicFunction::BoolToCString (bool thisBool) // Converts a bool to CString
{
	return (thisBool)?CString("TRUE"):CString("FALSE");
}

bool CBasicFunction::FileIsEmpty (std::string filePath) // Checks if a file is empty by peeking first char (non-existent files will mount empty)
{
	std::ifstream file (filePath.c_str(),std::ios::in);
	return (file.peek() == EOF);
}

unsigned CBasicFunction::CharsInFile (std::string filePath)
{
	if (CBasicFunction::FileIsEmpty(filePath))
		return 0;
	std::ifstream file(filePath.c_str());
	std::string total;
	std::string temp;
	while (!file.eof())
	{
		std::getline(file,temp);
		total += temp;
	}
	return unsigned(total.length());
}

std::string CBasicFunction::Truncate (std::string str, int x) // Truncates an std::string by x chars
{
	int newSize = int(str.size()-x);
	str.resize(newSize);
	return str;
}

bool CBasicFunction::StrAlphaCompare(std::string str1, std::string str2)
{
	// Returns true if str1 comes before str2 bibliographically
	for (unsigned i=0; i<str1.length() && i<str2.length(); i++)
		if (tolower(str1[i]) != tolower(str2[i]))
			return (tolower(str1[i]) < tolower(str2[i]));
	return true; // Strings are identical
}

/********************************************
*											*
*		Mikhail's basic and misc			*
*				functions					*
*				The end						*
*											*
********************************************/

//Read words from a string
vector<string> CBasicFunction::LoadWordsFromString(std::string sourcestring, vector<unsigned int>& wordfrequency)
{
	if(sourcestring.size()<=0)
	{
		vector<string> wordlist;
		return wordlist;
	}

	//Check if the last character is char or not, if yes, add an extra char \n char to the last
	bool charadded = false;
	if(IsLetter(sourcestring[sourcestring.size()-1]))
	{
		sourcestring.push_back('\n');
		charadded = true;
	}

	//Create the excluding list loading from the file
	vector<string> excludelist = CBasicFunction::GetExcludingList();

	vector<string> wordlist;
	//Loop to get the words
	for(int i=1; i<(int)sourcestring.size(); i++)
	{
		//Encountering the end of a word
		if(!IsLetter(sourcestring[i])&&IsLetter(sourcestring[i-1]))
		{
			//Search back for the beginning of the words
			int tempi = i-1;
			for(; tempi>=0; tempi--)
				if(!IsLetter(sourcestring[tempi]))
					break;
			tempi++;

			//Push the word to a string and reset to lower case
			string newstring(&(sourcestring.c_str()[tempi]), i-tempi);
			for(int j=0; j<(int)newstring.size(); j++)
			{
				if(newstring[j]<='Z'&&newstring[j]>='A')
					newstring[j] += ('a'-'A');
			}

			//Compare the string with the excluding list
			bool stringexcluded = false;
			for(int j=0; j<(int)excludelist.size(); j++)
			{
				if(excludelist[j].compare(newstring)==0)
				{
					stringexcluded = true;
					break;
				}
			}
			if(stringexcluded)
				continue;
			//Compare the string with the existing list
			bool stringexist = false;
			for(int j=0; j<(int)wordlist.size(); j++)
			{
				//If string existed, increase the counter by 1
				if(wordlist[j].compare(newstring)==0)
				{
					wordfrequency[j] ++;
					stringexist = true;
					break;
				}
			}
			//If not existed, push it into the wordlist
			if(!stringexist)
			{
				wordlist.push_back(newstring);
				wordfrequency.push_back((unsigned int)1);
			}
		}
	}
	
	//If \n char at last added, delete it
	if(charadded)
		sourcestring.resize(sourcestring.size()-1);

	return wordlist;
}

//Load words from a string by lines from a temporal file
vector<string> CBasicFunction::LoadWordsFromString(std::string sourcefile)
{
	std::ifstream sourcestream(sourcefile.c_str());
	vector<string> words;
	while(!sourcestream.eof())
	{
		string word;
		std::getline(sourcestream, word);
		words.push_back(word);
	}
	if(words.size()==0)
		throw string("No words loaded from the file");
	return words;
}

//Validate if a char is a letter or not
bool CBasicFunction::IsLetter(char singlechar)
{
	if(singlechar<='z' && singlechar>='a')
		return true;
	if(singlechar<='Z' && singlechar>='A')
		return true;
	return false;
}

//Locate the sentences that contains the word
void CBasicFunction::LocateSentence(std::string sourcestring, std::string lowercasestring,
									std::string word, std::vector<string>& sentences)
{
	if(lowercasestring.size()!=sourcestring.size())
	{
		return;
	}

	unsigned int i = 0;
	unsigned int sourcelength = (unsigned int)lowercasestring.size();
	unsigned int wordlength = (unsigned int)word.size();
	const char* sourcepointer = lowercasestring.c_str();
	const char* wordstart = word.c_str();

	while(i!=(unsigned int)-1)
	{
		//Find the next match
		i=CBasicFunction::SearchStr(wordstart, sourcepointer, ++i, wordlength, sourcelength);
		if(i!=(unsigned int)-1)
		{
			//Search forward and backward for the fullstop

			//Search forward and backward for the fullstop
			//string::size_type forwardstop = CBasicFunction::SearchForwardForStop(sourcestring, i);
			//string::size_type backwardstop = CBasicFunction::SearchBackwardForStop(sourcestring, i);
			unsigned int forwardstop = CBasicFunction::SearchForwardForStop(sourcepointer, sourcelength, i);
			unsigned int backwardstop = CBasicFunction::SearchBackwardForStop(sourcepointer, sourcelength, i);

			//Push the string between forwardstop and backwardstop into sentences
			//sentences.push_back(string(&(lowercasestring.c_str()[static_cast<unsigned int>(backwardstop)]),
			//	(static_cast<unsigned int>(forwardstop)-static_cast<unsigned int>(backwardstop))));
			sentences.push_back(string(&(sourcepointer[backwardstop]), forwardstop-backwardstop));

		}
		else
			break;
	}

	return;
}

//Locate the sentences that contains the word
void CBasicFunction::LocateSentence(std::string sourcestring, std::string word,
									std::vector<string>& sentences)
{
	unsigned int i = 0;
	unsigned int sourcelength = (unsigned int)sourcestring.size();
	unsigned int wordlength = (unsigned int)word.size();
	const char* sourcepointer = sourcestring.c_str();
	const char* wordstart = word.c_str();

	while(i!=(unsigned int)-1)
	{
		//Find the next match
		i=CBasicFunction::SearchStr(wordstart, sourcepointer, ++i, wordlength, sourcelength);
		if(i!=(unsigned int)-1)
		{
			//Search forward and backward for the fullstop

			//Search forward and backward for the fullstop
			//string::size_type forwardstop = CBasicFunction::SearchForwardForStop(sourcestring, i);
			//string::size_type backwardstop = CBasicFunction::SearchBackwardForStop(sourcestring, i);
			unsigned int forwardstop = CBasicFunction::SearchForwardForStop(sourcepointer, sourcelength, i);
			unsigned int backwardstop = CBasicFunction::SearchBackwardForStop(sourcepointer, sourcelength, i);

			//Push the string between forwardstop and backwardstop into sentences
			//sentences.push_back(string(&(lowercasestring.c_str()[static_cast<unsigned int>(backwardstop)]),
			//	(static_cast<unsigned int>(forwardstop)-static_cast<unsigned int>(backwardstop))));
			sentences.push_back(string(&(sourcepointer[backwardstop]), forwardstop-backwardstop));
		}
		else
			break;
	}
}

//Search forward a string for a fullstop
string::size_type CBasicFunction::SearchForwardForStop(std::string sourcestring, string::size_type startposition)
{
	string::size_type fullstop = startposition;
	while(fullstop!=sourcestring.npos)
	{
		//Search for the next fullstop
		string::size_type endofline = sourcestring.find_first_of('\r', fullstop);
		fullstop = sourcestring.find_first_of('.', ++fullstop);
		if(endofline<fullstop && endofline!=sourcestring.npos)
			return --endofline;
		//Validate if it is a fullstop
		if(fullstop==sourcestring.npos||fullstop>sourcestring.size()-3) //Full stop found, break;
		{
			fullstop = sourcestring.size()-1;
			break;
		}
		if(sourcestring[fullstop+1]=='\t'||sourcestring[fullstop+1]=='\n'||sourcestring[fullstop+1]
			=='\r') //Fullstop found, break;
			break;
		if(sourcestring[fullstop+1]==' '||(sourcestring[fullstop+2]<='A'&&sourcestring[fullstop+2]>='Z')) //Fullstop found, break
			break;
	}
	return fullstop;
}

//Search forward a string for a fullstop
unsigned int CBasicFunction::SearchForwardForStop(const char *sourcestring, 
												  unsigned int sourcelength, unsigned int startposition)
{
	unsigned int fullstop = startposition;
	while(fullstop!=(unsigned int)-1)
	{
		//Search for the next stop
		unsigned int endofline = CBasicFunction::SearchForward('\r', sourcestring, ++fullstop, sourcelength);
		fullstop = CBasicFunction::SearchForward('.', sourcestring, fullstop, sourcelength);
		if(endofline<fullstop&&endofline!=(unsigned int)-1)
			return --endofline;
		//Validate if it is a full stop
		if(fullstop==(unsigned int)-1||fullstop>=sourcelength-3)
		{
			fullstop = --sourcelength;
			break;
		}
		if(sourcestring[fullstop+1]=='\t'||sourcestring[fullstop+1]=='\n'||sourcestring[fullstop+1]=='\r')
			break;
		if(sourcestring[fullstop+1]==' '||(sourcestring[fullstop+2]<='A'&&sourcestring[fullstop+2]>='Z'))
			break;
	}
	return fullstop;
}

//Search backward a string for a fullstop
string::size_type CBasicFunction::SearchBackwardForStop(string sourcestring, string::size_type startposition)
{
	string::size_type fullstop = startposition;
	while(fullstop!=sourcestring.npos)
	{
		//Search for the next fullstop
		string::size_type endofline = sourcestring.find_last_of('\r', fullstop-1);
		fullstop = sourcestring.find_last_of('.', --fullstop);
		if(endofline>fullstop && endofline!=sourcestring.npos)
			return ++endofline;
		//Validate if it is a fullstop
		if(fullstop==sourcestring.npos||fullstop>sourcestring.size()-3) //Full stop found, break;
		{
			fullstop = static_cast<string::size_type>(0);
			break;
		}
		if(sourcestring[fullstop+1]=='\t'||sourcestring[fullstop+1]=='\n'||sourcestring[fullstop+1]=='\r') //Fullstop found, break;
		{
			fullstop ++;
			fullstop ++;
			break;
		}
		if(sourcestring[fullstop+1]==' '||(sourcestring[fullstop+2]<='A'&&sourcestring[fullstop+2]>='Z')) //Fullstop found, break
		{
			fullstop ++;
			fullstop ++;
			break;
		}
	}
	return fullstop;
}

unsigned int CBasicFunction::SearchBackwardForStop(const char* sourcestring, unsigned int sourcelength,
												   unsigned int startposition)
{
	unsigned int fullstop = startposition;
	while(fullstop!=(unsigned int)-1)
	{
		//Search for the next fullstop
		unsigned int endofline = CBasicFunction::SearchBackward('\r', sourcestring, --fullstop, sourcelength);
		fullstop = CBasicFunction::SearchBackward('.', sourcestring, fullstop, sourcelength);
		if(endofline>fullstop && endofline!=(unsigned int)-1)
			return ++endofline;
		//Validate if it is a fullstop
		if(fullstop==(unsigned int)-1||fullstop>sourcelength-3) //Full stop found, break;
		{
			fullstop = 0;
			break;
		}
		if(sourcestring[fullstop+1]=='\t'||sourcestring[fullstop+1]=='\n'||sourcestring[fullstop+1]=='\r') //Fullstop found, break;
		{
			fullstop +=2;
			break;
		}
		if(sourcestring[fullstop+1]==' '||(sourcestring[fullstop+2]<='A'&&sourcestring[fullstop+2]>='Z')) //Fullstop found, break
		{
			fullstop +=2;
			break;
		}
	}
	return fullstop;
}

//Change all the caps into lower cases
string CBasicFunction::ChangeToLowerCase(std::string sourcestring)
{
	string newstring = sourcestring;
	typedef string::size_type stringsize;
	stringsize sizeofstring = newstring.size();
	for(stringsize i=0; i<sizeofstring; i++)
		if(newstring[i]>='A'&&newstring[i]<='Z')
			newstring[i] += ('a' - 'A');
	return newstring;
}

//String into words
vector<string> CBasicFunction::StringToWords(std::string sourcestring)
{
	//Check if the string if empty
	if(sourcestring.size()<=0)
	{
		vector<string> wordlist;
		return wordlist;
	}

	//Check if the last character is char or not, if yes, add an extra char \n char to the last
	bool charadded = false;
	if(IsLetter(sourcestring[sourcestring.size()-1]))
	{
		sourcestring.push_back('\n');
		charadded = true;
	}

	vector<string> wordlist;
	//Loop to get the words
	for(int i=1; i<(int)sourcestring.size(); i++)
	{
		//Encountering the end of a word
		if(!IsLetter(sourcestring[i])&&IsLetter(sourcestring[i-1]))
		{
			//Search back for the beginning of the words
			int tempi = i-1;
			for(; tempi>=0; tempi--)
				if(!IsLetter(sourcestring[tempi]))
					break;
			if(!IsLetter(sourcestring[i]))
				tempi++;

			//Push the word to a string and reset to lower case
			string newstring(&(sourcestring.c_str()[tempi]), i-tempi);
			for(int j=0; j<(int)newstring.size(); j++)
			{
				if(newstring[j]<='Z'&&newstring[j]>='A')
					newstring[j] += ('a'-'A');
			}
			wordlist.push_back(newstring);
		}
	}
	
	//If \n char at last added, delete it
	if(charadded)
		sourcestring.resize(sourcestring.size()-1);

	return wordlist;
}

//String into words
vector<string> CBasicFunction::StringToWords(std::string sourcestring, std::string wholeword)
{
	//Check if the string if empty
	if(sourcestring.size()<=0)
	{
		vector<string> wordlist;
		return wordlist;
	}

	//Check if the string if empty
	if(sourcestring.size()<=0)
	{
		throw string("String is empty!");
	}

	vector<string> wordlist;

	//Find the wholeword and exclude it from the string
	string::size_type wordlocation = CBasicFunction::FindLastWordInString(sourcestring, wholeword);
	vector<string> wordsbefore;
	//Get the words before and after the wholeword
	if(wordlocation!=sourcestring.npos)
	{
		//Convert the front part
		wordsbefore = StringToWords(string(&(sourcestring.c_str()[0]),
			static_cast<unsigned int>(wordlocation)), wholeword);
		wordlist.insert(wordlist.end(), wordsbefore.begin(), wordsbefore.end());

		wordlist.insert(wordlist.end(), wholeword);
		vector<string> wordsafter = StringToWords(string(&(sourcestring.c_str()[static_cast<unsigned int>(wordlocation)+wholeword.size()])));
		wordlist.insert(wordlist.end(), wordsafter.begin(), wordsafter.end());
	}
	else
	{
		wordsbefore = StringToWords(sourcestring);
		wordlist.insert(wordlist.end(), wordsbefore.begin(), wordsbefore.end());
	}

	return wordlist;
}

//Collocations
void CBasicFunction::CollocationSentences(std::vector<string> sentences, std::string word,
								 std::vector<string> &collocations,
								 std::vector<vector<double>> &collocationfrequency,
								 std::vector<double> &collocationsumfrequency,
								 std::vector<double> &collocationspread)
{
	//Get the excluding list
	vector<string> excludelist = CBasicFunction::GetExcludingList();
	excludelist.push_back(word);	//Exclude the word itself from the list

	//Get the frequency for everyword in each sentence
	for(unsigned int i=0; i<sentences.size(); i++)
	{
		//Convert the sentences into words and get each collocations
		vector<string> wordsinsentence = CBasicFunction::StringToWords(sentences[i], word);

		//Location of the word in the sentence
		vector<unsigned int> wordlocations;
		for(unsigned int j=0; j<wordsinsentence.size(); j++)
		{
			if(word.compare(wordsinsentence[j])==0)
				wordlocations.push_back(j);
		}

		//For each word location
		for(unsigned int j=0; j<wordlocations.size(); j++)
		{
			//Get the collocations
			unsigned int startposition = (unsigned int)max((int)0, (int)wordlocations[j]-(int)5);
			unsigned int endposition = (unsigned int)min((int)wordsinsentence.size()-1, (int)wordlocations[j]+(int)5);
			unsigned int indicator = (unsigned int)(5+startposition-wordlocations[j]);
			for(unsigned int k=startposition; k<=endposition; k++)
			{
				if(k==wordlocations[j])
					continue;

				//Check if the word in excluding list or not
				bool wordinexcludelist = false;
				for(unsigned int l=0; l<excludelist.size(); l++)
					if(excludelist[l].compare(wordsinsentence[k])==0)
					{
						wordinexcludelist = true;
						break;
					}
				if(wordinexcludelist)
				{
					indicator ++;
					continue;
				}

				//Check if the collocations in the list or not
				bool collocationexisted = false;
				for(unsigned int l=0; l<collocations.size(); l++)
				{
					if(collocations[l].compare(wordsinsentence[k])==0)	//If in the list
					{
						collocationexisted = true;
						collocationfrequency[l][indicator] += 1.0;
						break;
					}
				}
				if(!collocationexisted)	//If not
				{
					collocations.push_back(wordsinsentence[k]);
					vector<double> wordfrequency;
					wordfrequency.resize(10);
					for(unsigned int l=0; l<10; l++)
						wordfrequency[l] = (double)0.0;
					wordfrequency[indicator] += 1;
					collocationfrequency.push_back(wordfrequency);
				}
				indicator ++;
			}
		}
	}

	//Compute the measurements for each collocations
	collocationsumfrequency.resize(collocations.size());
	collocationspread.resize(collocations.size());
	for(unsigned int i=0; i<collocationfrequency.size(); i++)
	{
		double total = 0.0;
		for(int k=0; k<10; k++)
			total += collocationfrequency[i][k];
		collocationsumfrequency[i] = total;
		total = 0.0;
		for(int k=0; k<10; k++)
			total += (collocationfrequency[i][k]-collocationsumfrequency[i]/(double)10.0)*(collocationfrequency[i][k]-collocationsumfrequency[i]/(double)10.0);
		collocationspread[i] = total/10.0;
	}

	return;
}

//Get the excluding list
vector<string> CBasicFunction::GetExcludingList()
{
	vector<string> excludelist;
	//Read the file content;
	string filepath = string(HOMEDIRECTORY)+string("exclude.txt");
	std::ifstream excludefile (filepath.c_str());
	//Read each line in the file
	while(!excludefile.eof())
	{
		string temporalstring;
		std::getline(excludefile, temporalstring);
		excludelist.push_back(temporalstring);
	}
	return excludelist;
}

//Double matrix to string
string CBasicFunction::DoubleMatrixToString(double **doubledata, int xdimension, int ydimension, int precision, char separator)
{
	std::ostringstream systemstring;
	systemstring.precision(precision);
	for(int i=0; i<xdimension; i++)
	{
		for(int j=0; j<ydimension; j++)
		{
			systemstring << doubledata[i][j] << separator;
		}
		systemstring << "\r\n";
	}
	return systemstring.str();
}

//Get TFIDF
vector<vector<double>> CBasicFunction::GetTFIDF(std::vector<string> words, std::vector<string> documents)
{
	//Initialize the vector
	vector<vector<double>> termfrequency;
	vector<double> inversedocumentfrequency;
	termfrequency.resize(words.size());
	inversedocumentfrequency.resize(words.size());
	for(unsigned int i=0; i<termfrequency.size(); i++)
	{
		termfrequency[i].resize(documents.size());
		for(unsigned int j=0; j<termfrequency[i].size(); j++)
			termfrequency[i][j] = 0.0;
		inversedocumentfrequency[i] = 0.0;
	}

	//Get the frequency of each term in each document
	for(unsigned int i=0; i<documents.size(); i++)
	{
		for(unsigned int j=0; j<words.size(); j++)
		{
			string::size_type wordposition = static_cast<string::size_type>(documents[i].size());
			while(wordposition!=documents[i].npos)
			{
				wordposition = documents[i].rfind(words[j], --wordposition);
				if(wordposition!=documents[i].npos)	//The word found
				{	//Validate if it is a full word or part of another word
					if(wordposition-1>=0&&(wordposition+words[j].size())<documents[i].size())
					{
						if(IsLetter(documents[i][wordposition-1])||IsLetter(documents[i][wordposition+words[j].size()]))
							continue;
						else
							termfrequency[j][i] += 1.0;
					}
					else
						termfrequency[j][i] += 1.0;
				}
			}
		}
	}

	//Get the document frequency and inverse document frequency
	vector<double> documentfrequency;
	documentfrequency.resize(documents.size());
	for(unsigned int i=0; i<documents.size(); i++)
	{
		documentfrequency[i] = 0.0;
		for(unsigned int j=0; j<words.size(); j++)
		{
			documentfrequency[i] += termfrequency[j][i];
		}
	}
	for(unsigned int i=0; i<words.size(); i++)
	{
		for(unsigned int j=0; j<documents.size(); j++)
		{
			if(termfrequency[i][j]>0)
				inversedocumentfrequency[i] += 1.0;
		}
	}

	//Compute the TFIDF
	vector<vector<double>> tfidf;
	tfidf.resize(words.size());
	for(unsigned int i=0; i<words.size(); i++)
	{
		tfidf[i].resize(documents.size());
		for(unsigned int j=0; j<documents.size(); j++)
		{
			tfidf[i][j] = termfrequency[i][j]/documentfrequency[j]*
				(log((double)documents.size()/(double)inversedocumentfrequency[i])+(double)1.0);
		}
	}

	return tfidf;
}

//Analyse a set of documents
void CBasicFunction::AnalyseDocuments(std::vector<string> documents, std::vector<string> &words,
									  std::vector<unsigned int> &wordfrequency,
									  std::vector<vector<double>>& tfidf)
{
	//Fuse all the documents
	string alldocumentstring;
	for(unsigned int i=0; i<documents.size(); i++)
	{
		alldocumentstring += string(documents[i].c_str());
		alldocumentstring += "\r\n";
	}
	words = LoadWordsFromString(alldocumentstring, wordfrequency);

	//Get the tfidf
	tfidf = GetTFIDF(words, documents);

	return;
}

//Load the BNC corpus
void CBasicFunction::LoadBNCCorpus(std::vector<string> &words, std::vector<unsigned int> &frequency,
								   unsigned long& sumoffrequency)
{
	//Load the file
	string filepath = string(HOMEDIRECTORY)+string("BNC.freq");
	std::ifstream bncfile(filepath.c_str());
	string sumof;
	std::getline(bncfile, sumof);
	sumoffrequency = (unsigned long)atol(sumof.c_str());
	while(!bncfile.eof())
	{
		//Get each line
		string eachline;
		std::getline(bncfile, eachline);
		string::size_type separator = eachline.find_first_of(',');
		if(separator!=eachline.npos)
		{
			//Push the string to words
			words.push_back(string(eachline.c_str(), static_cast<unsigned int>(separator)));
			//Convert the number to unsigned int
			unsigned int currentfrequency = (unsigned int)atoi(&(eachline.c_str()[++separator]));
			frequency.push_back(currentfrequency);
		}
	}
	return;
}

//Compute the weirdness
vector<double> CBasicFunction::ComputerWeirdness(std::vector<string> words, std::vector<unsigned int> frequency)
{
	//Load the BNC corpus
	vector<string> bnccorpus;
	vector<unsigned int> bncfrequency;
	unsigned long sumofbncfrequency;
	CBasicFunction::LoadBNCCorpus(bnccorpus, bncfrequency, sumofbncfrequency);

	//Compute the sum of frequency
	unsigned long sumoffrequency = 0;
	for(unsigned int i=0; i<frequency.size(); i++)
		sumoffrequency += frequency[i];

	//Compute the weirdness
	vector<double> weirdness;
	weirdness.resize(words.size());
	for(unsigned int i=0; i<words.size(); i++)
	{
		//Find the word in the BNC corpus
		for(unsigned int j=0; j<bnccorpus.size(); j++)
		{
			if(words[i].compare(bnccorpus[j])==0)	//If the word existed in the bnc corpus, use it
			{
				weirdness[i] = ((double)frequency[i]/(double)sumoffrequency)/
					((double)bncfrequency[j]/(double)sumofbncfrequency);
				break;
			}
		}
		//If not, set its bnc frequency to 1
		weirdness[i] = ((double)frequency[i]/(double)sumoffrequency)/
			((double)1.0/(double)sumofbncfrequency);
	}
	return weirdness;
}

//Do the collocations for a set of words
vector<unsigned int> CBasicFunction::Collocations(std::string sourcestring, std::vector<string> wordlist,
								  std::vector<string>& collocations,
								  std::vector<vector<double>> &frequency,
								  std::vector<double> &spread, std::vector<double> &sumfrequency)
{
	vector<unsigned int> collocationcounter;
	string lowerstring = CBasicFunction::ChangeToLowerCase(sourcestring);
	//Do the collocations for each word and fuse the results
	for(unsigned int i=0; i<wordlist.size(); i++)
	{
		//Locate the sentences
		vector<string> sentences;
		CBasicFunction::LocateSentence(sourcestring, lowerstring, wordlist[i], sentences);

		//Do collocations for the word
		vector<string> eachcollocation;
		vector<vector<double>> eachfrequency;
		vector<double> eachspread;
		vector<double> eachsum;
		CBasicFunction::CollocationSentences(sentences, wordlist[i], eachcollocation,
			eachfrequency,	eachsum, eachspread);
		collocations.insert(collocations.end(), eachcollocation.begin(), eachcollocation.end());
		frequency.insert(frequency.end(), eachfrequency.begin(), eachfrequency.end());
		spread.insert(spread.end(), eachspread.begin(), eachspread.end());
		sumfrequency.insert(sumfrequency.end(), eachsum.begin(), eachsum.end());
		collocationcounter.push_back((unsigned int)collocations.size());
	}

	return collocationcounter;
}

//Select the keyterms
vector<string> CBasicFunction::SelectKeyterms(std::vector<string> wordlist,	  std::vector<unsigned int> frequency,
											  std::vector<double> weirdness, unsigned int ignorefrequency)
{
	double threshold = 0.0;
	unsigned int sizeofword = 4;
	//Validate the data
	if(wordlist.size()!=frequency.size()||frequency.size()!=weirdness.size())
		throw string("Size of wordlist, frequency, weirdness not equal!");
	if(wordlist.size()==0)
		throw string("No word in wordlist!");
	//Calculate the standard deviation and average of the frequency and weirdness
	double meanfrequency = 0;
	double stdfrequency = 0;
	double meanweirdness = 0;
	double stdweirdness = 0;
	unsigned int counter = 0;	//A counter for ignoring the terms with frequency less than 5

	//Compute the average frequency and weirdness
	for(unsigned int i=0; i<wordlist.size(); i++)
	{
		//If frequency is less than ignorefrequency, or if the word is less than sizeofword, skip the word
		if(frequency[i]<=ignorefrequency||wordlist[i].size()<sizeofword)
			continue;

		meanfrequency += (double)frequency[i];
		meanweirdness += weirdness[i];
		counter ++;
	}
	meanfrequency /= (double)counter;
	meanweirdness /= (double)counter;

	//Computer the stardard deviation
	for(unsigned int i=0; i<wordlist.size(); i++)
	{
		//If frequency is less than ignorefrequency, or if the word is less than sizeofword, skip the word
		if(frequency[i]<=ignorefrequency||wordlist[i].size()<sizeofword)
			continue;

		stdfrequency += ((double)frequency[i]-meanfrequency)*((double)frequency[i]-meanfrequency);
		stdweirdness += (weirdness[i]-meanweirdness)*(weirdness[i]-meanweirdness);
	}
	stdfrequency = sqrt(stdfrequency/(double)(wordlist.size()-1));
	stdweirdness = sqrt(stdweirdness/(double)(wordlist.size()-1));

	//Get the significant keyterms
	vector<string> keywords;
	for(unsigned int i=0; i<wordlist.size(); i++)
	{
		//If frequency is less than ignorefrequency, or if the word is less than sizeofword, skip the word
		if(frequency[i]<=ignorefrequency||wordlist[i].size()<sizeofword)
			continue;

		if(((double)frequency[i]-meanfrequency)/stdfrequency>threshold&&
			(weirdness[i]-meanweirdness)/stdweirdness>threshold)
		{
			keywords.push_back(wordlist[i]);
		}
	}

	return keywords;
}

//Find the position of the last occurrence of the word in the string
string::size_type CBasicFunction::FindLastWordInString(std::string sourcestring, std::string word)
{
	string::size_type lastposition = sourcestring.rfind(word);
	if(lastposition==sourcestring.npos)
		return lastposition;
	else
	{
		//Test the char before
		bool before = false;
		if(lastposition==0)
			before = true;
		else if(!IsLetter(sourcestring[lastposition-1]))
			before = true;
		//Test the char after
		bool after = false;
		if(lastposition+word.size()==sourcestring.size())
			after = true;
		else if(!IsLetter(sourcestring[lastposition+word.size()]))
			after = true;
		if(before&&after)
			return lastposition;
		else
			lastposition = CBasicFunction::FindLastWordInString(string(sourcestring.c_str(), static_cast<unsigned int>(lastposition)), word);
	}
	return lastposition;
}

//Find the position of the last occurrence of the word in the string within the range start~end
string::size_type CBasicFunction::FindLastWordInString(std::string sourcestring, std::string word, 
													   string::size_type start, string::size_type end)
{
	string newstring = string(&(sourcestring.c_str()[start]), static_cast<unsigned int>(end-start));
	return CBasicFunction::FindLastWordInString(newstring, word);
}

//Convert a double pointer to double vector
vector<double> CBasicFunction::PointerToVector(double *doublepointer, unsigned int numberofdata)
{
	vector<double> doublevector;
	doublevector.resize(numberofdata);
	for(unsigned int i=0; i<numberofdata; i++)
		doublevector[i] = doublepointer[i];
	return doublevector;
}

//Convert a double pointer to a double vector
vector<vector<double>> CBasicFunction::PointerToVector(double **doublepointer, unsigned int sizex, unsigned int sizey)
{
	vector<vector<double>> doublevector;
	doublevector.resize(sizex);
	for(unsigned int i=0; i<sizex; i++)
		doublevector[i] = CBasicFunction::PointerToVector(doublepointer[i], sizey);
	return doublevector;
}

//Convert a vector into a pointer
double* CBasicFunction::VectorToPointer(std::vector<double> doublevector, unsigned int &numberofdata)
{
	numberofdata = (unsigned int)doublevector.size();
	double* datapointer = new double[numberofdata];
	for(unsigned int i=0; i<numberofdata; i++)
		datapointer[i] = doublevector[i];
	return datapointer;
}

//Convert a vector<vector> into a pointer
double* CBasicFunction::VectorToPointer(std::vector<vector<double> > doublevector, unsigned int &numberofdata)
{
	//Get the number of data	
	numberofdata = 0;
	for(unsigned int i=0; i<(unsigned int)doublevector.size(); i++)
		numberofdata += (unsigned int)doublevector[i].size();

	//Get the data
	double* datapointer = new double[numberofdata];
	unsigned int counter = 0;
	for(unsigned int i=0; i<(unsigned int)doublevector.size(); i++)
		for(unsigned int j=0; j<(unsigned int)doublevector[i].size(); j++)
			datapointer[counter++] = doublevector[i][j];

	return datapointer;
}

//Convert a vector<vector>> into a double pointer
double** CBasicFunction::VectorToPointer(vector<vector<double>> doublevector, unsigned int& dimension1, unsigned int& dimension2)
{
	//Obtain the size, varify the size, and create memory
	dimension1 = (unsigned int)doublevector.size();
	if(dimension1 == 0)
		throw string("Vector empty");
	dimension2 = (unsigned int)doublevector[0].size();
	if(dimension2 == 0)
		throw string("Each vector is empty");
	double** doublepointer = new double*[dimension1];

	//Convert the vector one by one
	for(unsigned int i=0; i<dimension1; i++)
	{
		unsigned int numberofdata;
		doublepointer[i] = CBasicFunction::VectorToPointer(doublevector[i], numberofdata);
		if(numberofdata!=dimension2)
			throw string("Size of vector not equal");
	}
	return doublepointer;
}

//Load images from a folder including all the subfolders
vector<string> CBasicFunction::LoadImagesFromAFolder()
{
	// Parameters for a folder view
	LPWSTR lpszDir = new WCHAR[_MAX_PATH];
	LPWSTR buffer = new WCHAR[_MAX_PATH];

	::OleInitialize(NULL);
	BROWSEINFO br_info;
	memset(&br_info, 0, sizeof(br_info));

	// Browse the path for images
	br_info.ulFlags = BIF_USENEWUI;
	br_info.hwndOwner = NULL;
	br_info.lpszTitle = _T("Please Select a Folder for Images");

	LPITEMIDLIST pidl;
	pidl = ::SHBrowseForFolder(&br_info);

	if(pidl != NULL)
	{
		// Create a buffer to store the path and then get the path
		if(::SHGetPathFromIDList(pidl, lpszDir) != 0)
		{
			// Set the string value
		}
		// Free the item ID list
		CoTaskMemFree(pidl);
	}
	else
	{
		throw string("No folder selected");
	}
	::OleUninitialize();

	//Construct a list of folders
	vector<string> listoffolders;
	CT2CA asciistring(lpszDir);
	listoffolders.push_back(string(asciistring));

	vector<string> listofimages;

	unsigned int i=0;
	do
	{
		//Construct the searching
		_finddata_t data;
		string fpath = listoffolders[i]+"\\*.*";
		long handle = (long)_findfirst(fpath.c_str(), &data);
		//If something found
		if(handle >= 0)
		{
			do
			{
				if(data.attrib & _A_SUBDIR)	//If it is a folder, push it to listoffolders
				{
					if(strcmp(data.name, ".")!=0 && strcmp(data.name, "..")!=0)
						listoffolders.push_back(listoffolders[i] + "\\" + data.name);
				}
				else	//If it is a file, examine if it is an image or not
				{
					//Check the file extension
					string filename(data.name);
					string fileextension(&(filename.c_str()[filename.size()-4]));
					if(fileextension.compare(string(".bmp"))==0 || fileextension.compare(string(".BMP"))==0
						|| fileextension.compare(string(".tif"))==0 || fileextension.compare(string(".TIF"))==0
						|| fileextension.compare(string(".jpg"))==0 || fileextension.compare(string("JPG"))==0)
					{
						listofimages.push_back(listoffolders[i]+string("\\")+string(filename));
					}
				}
			}while(_findnext(handle, &data)==0);
		}
		_findclose(handle);
		i++;
	}while(i<listoffolders.size());
	
	if(listofimages.size()==0)
		throw string("No images found");

	return listofimages;
}

//Search for images in a folder
vector<string> CBasicFunction::LoadImagesFromAFolder(string folderpath)
{
	//Construct a list of folders
	vector<string> listoffolders;
	listoffolders.push_back(folderpath);

	vector<string> listofimages;

	unsigned int i=0;
	do
	{
		//Construct the searching
		_finddata_t data;
		string fpath = listoffolders[i]+"\\*.*";
		long handle = (long)_findfirst(fpath.c_str(), &data);
		//If something found
		if(handle >= 0)
		{
			do
			{
				if(data.attrib & _A_SUBDIR)	//If it is a folder, push it to listoffolders
				{
					if(strcmp(data.name, ".")!=0 && strcmp(data.name, "..")!=0)
						listoffolders.push_back(listoffolders[i] + "\\" + data.name);
				}
				else	//If it is a file, examine if it is an image or not
				{
					//Check the file extension
					string filename(data.name);
					string fileextension(&(filename.c_str()[filename.size()-4]));
					if(fileextension.compare(string(".bmp"))==0 || fileextension.compare(string(".BMP"))==0
						|| fileextension.compare(string(".tif"))==0 || fileextension.compare(string(".TIF"))==0
						|| fileextension.compare(string(".jpg"))==0 || fileextension.compare(string("JPG"))==0
						|| fileextension.compare(string(".tif"))==0 || fileextension.compare(string(".TIF"))==0)
					{
						listofimages.push_back(listoffolders[i]+string("\\")+string(filename));
					}
				}
			}while(_findnext(handle, &data)==0);
		}
		_findclose(handle);
		i++;
	}while(i<listoffolders.size());
	
	if(listofimages.size()==0)
		throw string("No images found");

	return listofimages;
}

//Pop up a dialog to select multiple file
vector<string> CBasicFunction::SelectMultipleFiles(CString stringfilter)
{
	vector<string> multiplefiles;
	//Pop up a dialog to select the files
	CFileDialog filedialog(TRUE, NULL, NULL, OFN_FILEMUSTEXIST|OFN_ALLOWMULTISELECT, stringfilter);
	//Display the dialog to load image files
	TCHAR* filestructure = new TCHAR[2048*10];
	filedialog.m_ofn.lpstrFile = filestructure;
	filestructure[0] = '\0';
	filedialog.m_ofn.nMaxFile = 2048*10;
	CString dialogcaption("Select multiple files...");
	filedialog.m_ofn.lpstrTitle = dialogcaption;
	HRESULT m_hResult = (int)filedialog.DoModal();
	if(FAILED(m_hResult))
		throw string("Failed to open dialog");

	//Inseart the files into the image list
	POSITION fileposition = filedialog.GetStartPosition();
	while(fileposition!=NULL)
	{
		//Inseart the file name into the list
		CString filepath = filedialog.GetNextPathName(fileposition);
		CT2CA asciistring(filepath);
		string filepatha(asciistring);
		multiplefiles.push_back(filepatha);
	}
	delete filestructure;

	return multiplefiles;
}

//select multiple files
vector<string> CBasicFunction::SelectMultipleFiles(CString stringfilter, CString dialogcaption)
{
	vector<string> multiplefiles;
	//Pop up a dialog to select the files
	CFileDialog filedialog(TRUE, NULL, NULL, OFN_FILEMUSTEXIST|OFN_ALLOWMULTISELECT, stringfilter);
	//Display the dialog to load image files
	TCHAR* filestructure = new TCHAR[2048*10];
	filedialog.m_ofn.lpstrFile = filestructure;
	filestructure[0] = '\0';
	filedialog.m_ofn.nMaxFile = 2048*10;
	filedialog.m_ofn.lpstrTitle = dialogcaption;
	HRESULT m_hResult = (int)filedialog.DoModal();
	if(FAILED(m_hResult))
		throw string("Failed to open dialog");

	//Inseart the files into the image list
	POSITION fileposition = filedialog.GetStartPosition();
	while(fileposition!=NULL)
	{
		//Inseart the file name into the list
		CString filepath = filedialog.GetNextPathName(fileposition);
		CT2CA asciistring(filepath);
		string filepatha(asciistring);
		multiplefiles.push_back(filepatha);
	}
	delete filestructure;

	return multiplefiles;
}

//Construct a dialog to select a folder
string CBasicFunction::SelectAFolder()
{
	// Parameters for a folder view
	LPWSTR lpszDir = new WCHAR[_MAX_PATH];
	LPWSTR buffer = new WCHAR[_MAX_PATH];

	::OleInitialize(NULL);
	BROWSEINFO br_info;
	memset(&br_info, 0, sizeof(br_info));

	// Browse the path for images
	br_info.ulFlags = BIF_USENEWUI;
	br_info.hwndOwner = NULL;
	br_info.lpszTitle = _T("Please Select a Folder for Images");

	LPITEMIDLIST pidl;
	pidl = ::SHBrowseForFolder(&br_info);

	if(pidl != NULL)
	{
		// Create a buffer to store the path and then get the path
		if(::SHGetPathFromIDList(pidl, lpszDir) != 0)
		{
			// Set the string value
		}
		// Free the item ID list
		CoTaskMemFree(pidl);
	}
	else
	{
		throw string("No folder selected");
	}
	::OleUninitialize();

	//Convert the folder to string
	CT2CA asciistring(lpszDir);
	delete[] lpszDir;
	delete[] buffer;
	return string(asciistring);
}

//Select files from a folder
vector<string> CBasicFunction::GetFilesFromAFolder(std::string folderpath, std::string extension)
{
	//Construct a list of folders
	vector<string> listoffolders;
	listoffolders.push_back(folderpath);

	vector<string> listoffiles;

	string extensionlower = CBasicFunction::ChangeToLowerCase(extension);

	unsigned int i=0;
	do
	{
		//Construct the searching
		_finddata_t data;
		string fpath = listoffolders[i]+"\\*.*";
		long handle = (long)_findfirst(fpath.c_str(), &data);
		//If something found
		if(handle >= 0)
		{
			do
			{
				if(data.attrib & _A_SUBDIR)	//If it is a folder, push it to listoffolders
				{
					if(strcmp(data.name, ".")!=0 && strcmp(data.name, "..")!=0)
						listoffolders.push_back(listoffolders[i] + "\\" + data.name);
				}
				else	//If it is a file, examine if it is an image or not
				{
					//Check the file extension
					string filename(data.name);
					string fileextension;
					string::size_type pos = filename.find_last_of('.');
					if(pos == string::npos)
						fileextension = string("");
					else
						fileextension = string(&(filename.c_str()[pos+1]));
					if(fileextension.compare(extension)==0)
					{
						listoffiles.push_back(listoffolders[i]+string("\\")+string(filename));
					}
				}
			}while(_findnext(handle, &data)==0);
		}
		_findclose(handle);
		i++;
	}while(i<listoffolders.size());

	return listoffiles;
}

//Save data as binary file
void	CBasicFunction::SaveDataAsBinary(std::vector<vector<vector<double>>> data, string filepath)
{
	if(data.size()==0)
		throw string("No data in the vector");
	else if(data[0].size()==0)
		throw string("No data in the vector");
	unsigned int size1 = (unsigned int)data.size();
	unsigned int size2 = (unsigned int)data[0].size();
	unsigned int size3 = (unsigned int)data[0][0].size();
	//Create the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeCreate|CFile::modeWrite, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Write the header
	cfile.Write(&size1, sizeof(unsigned int));
	cfile.Write(&size2, sizeof(unsigned int));
	cfile.Write(&size3, sizeof(unsigned int));

	//Write the file content
	try
	{
		for(unsigned int i=0; i<size1; i++)
		{
			if(data[i].size()!=size2)
				throw string("Data dimension not equal");
			for(unsigned int j=0; j<size2; j++)
			{
				if(data[i][j].size()!=size3)
					throw string("Data dimension not equal");
				unsigned int numberofdata = (unsigned int)data[i][j].size();
				double* currentdata = VectorToPointer(data[i][j], numberofdata);
				cfile.Write(currentdata, numberofdata*sizeof(double));
				delete[] currentdata;
			}
		}
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	cfile.Close();
	return;
}

//Save binary data as file
void CBasicFunction::SaveDataAsBinary(vector<vector<vector<vector<double>>>> data, string filepath)
{
	if(data.size()==0)
		throw string("No data in the vector");
	else if(data[0].size()==0)
		throw string("No data in the vector");
	else if(data[0][0].size()==0)
		throw string("No data in the vector");
	unsigned int size1 = (unsigned int)data.size();
	unsigned int size2 = (unsigned int)data[0].size();
	unsigned int size3 = (unsigned int)data[0][0].size();
	unsigned int size4 = (unsigned int)data[0][0][0].size();

	//Create the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeCreate|CFile::modeWrite, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Write the header
	cfile.Write(&size1, sizeof(unsigned int));
	cfile.Write(&size2, sizeof(unsigned int));
	cfile.Write(&size3, sizeof(unsigned int));
	cfile.Write(&size4, sizeof(unsigned int));

	//Write the file content
	try
	{
		for(unsigned int i=0; i<size1; i++)
		{
			if(data[i].size()!=size2)
				throw string("Data dimension not equal");
			for(unsigned int j=0; j<size2; j++)
			{
				if(data[i][j].size()!=size3)
					throw string("Data dimension not equal");
				for(unsigned int k=0; k<size3; k++)
				{
					unsigned int numberofdata = (unsigned int)data[i][j][k].size();
					double* currentdata = VectorToPointer(data[i][j][k], numberofdata);
					cfile.Write(currentdata, numberofdata*sizeof(double));
					delete[] currentdata;
				}
			}
		}
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	cfile.Close();
	return;
}

//Save binary data as filie
void CBasicFunction::SaveDataAsBinary(std::vector<vector<double> > data, std::string filepath)
{
	if(data.size()==0)
		throw string("No data in the vector");
	else if(data[0].size()==0)
		throw string("No data in the vector");
	unsigned int size1 = (unsigned int)data.size();
	unsigned int size2 = (unsigned int)data[0].size();

	//Create the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeCreate|CFile::modeWrite, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Write the header
	cfile.Write(&size1, sizeof(unsigned int));
	cfile.Write(&size2, sizeof(unsigned int));

	//Write the file content
	try
	{
		for(unsigned int i=0; i<size1; i++)
		{
			if(data[i].size()!=size2)
				throw string("Data dimension not equal");
			unsigned int numberofdata = (unsigned int)data[i].size();
			double* currentdata = VectorToPointer(data[i], numberofdata);
			cfile.Write(currentdata, numberofdata*sizeof(double));
			delete[] currentdata;
		}
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	cfile.Close();
	return;
}

//Save data as binary file
void CBasicFunction::SaveDataAsBinary(vector<double> data, string filepath)
{
	if(data.size()==0)
		throw string("No data in the vector");
	unsigned int size1 = (unsigned int)data.size();

	//Create the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeCreate|CFile::modeWrite, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Write the header
	cfile.Write(&size1, sizeof(unsigned int));

	//Write the file content
	try
	{
		unsigned int numberofdata;
		double* currentdata = VectorToPointer(data, numberofdata);
		cfile.Write(currentdata, numberofdata*sizeof(double));
		delete[] currentdata;
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	cfile.Close();
	return;
}

//Load data from binary file
void CBasicFunction::LoadBinaryData(vector<double>& data, string filepath)
{
	//Open the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeRead, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}
	
	//Load the header
	unsigned int byteread;
	unsigned int size1;
	try
	{
		byteread = cfile.Read(&size1, sizeof(unsigned int));
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Load the data
	data.resize(size1);

	//Create a pointer to read the data
	double* currentdata = new double[size1];
	try
	{
		cfile.Read(currentdata, sizeof(double)*size1);
		data = PointerToVector(currentdata, size1);
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	cfile.Close();
	return;
}

//Load data from binary file
vector<vector<vector<double>>> CBasicFunction::LoadBinaryData(std::string filepath)
{
	//Open the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeRead, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}
	
	//Load the header
	unsigned int byteread;
	vector<vector<vector<double>>> data;
	unsigned int size1, size2, size3;
	try
	{
		byteread = cfile.Read(&size1, sizeof(unsigned int));
		byteread = cfile.Read(&size2, sizeof(unsigned int));
		byteread = cfile.Read(&size3, sizeof(unsigned int));
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Load the data
	data.resize(size1);
	for(unsigned int i=0; i<size1; i++)
	{
		data[i].resize(size2);
		for(unsigned int j=0; j<size2; j++)
		{
			//Create a pointer to read the data
			double* currentdata = new double[size3];
			try
			{
				cfile.Read(currentdata, sizeof(double)*size3);
			}
			catch(CFileException* e)
			{
				TCHAR errormessage[1024];
				e->GetErrorMessage(errormessage, 1024);
				CT2CA ascii(errormessage);
				string errorstring(ascii);
				throw errorstring;
			}
			//Convert the data as vector and append to the vector<vector<vector<double>>>
			data[i][j] = PointerToVector(currentdata, size3);
			delete[] currentdata;
		}
	}

	cfile.Close();
	return data;
}

//Load data from a binary file
void CBasicFunction::LoadBinaryData(vector<vector<double>>& data, string filepath)
{
	//Open the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeRead, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}
	
	//Load the header
	unsigned int byteread;
	unsigned int size1, size2;
	try
	{
		byteread = cfile.Read(&size1, sizeof(unsigned int));
		byteread = cfile.Read(&size2, sizeof(unsigned int));
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Load the data
	data.resize(size1);
	for(unsigned int i=0; i<size1; i++)
	{
		data[i].resize(size2);
		//Create a pointer to read the data
		double* currentdata = new double[size2];
		try
		{
			cfile.Read(currentdata, sizeof(double)*size2);
		}
		catch(CFileException* e)
		{
			TCHAR errormessage[1024];
			e->GetErrorMessage(errormessage, 1024);
			CT2CA ascii(errormessage);
			string errorstring(ascii);
			throw errorstring;
		}
		//Convert the data as vector and append to the vector<vector<vector<double>>>
		data[i] = PointerToVector(currentdata, size2);
		delete[] currentdata;
	}

	cfile.Close();
	return;
}

//Load binary data from file
void CBasicFunction::LoadBinaryData(vector<vector<vector<vector<double>>>>& data, string filepath)
{
	//Open the file
	CFile cfile;
	CFileException e;
	if(!cfile.Open(CString(filepath.c_str()), CFile::modeRead, &e))
	{
		TCHAR errormessage[1024];
		e.GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}
	
	//Load the header
	unsigned int byteread;
	unsigned int size1, size2, size3, size4;
	try
	{
		byteread = cfile.Read(&size1, sizeof(unsigned int));
		byteread = cfile.Read(&size2, sizeof(unsigned int));
		byteread = cfile.Read(&size3, sizeof(unsigned int));
		byteread = cfile.Read(&size4, sizeof(unsigned int));
	}
	catch(CFileException* e)
	{
		TCHAR errormessage[1024];
		e->GetErrorMessage(errormessage, 1024);
		CT2CA ascii(errormessage);
		string errorstring(ascii);
		throw errorstring;
	}

	//Load the data
	data.resize(size1);
	for(unsigned int i=0; i<size1; i++)
	{
		data[i].resize(size2);
		for(unsigned int j=0; j<size2; j++)
		{
			data[i][j].resize(size3);
			for(unsigned int k=0; k<size3; k++)
			{
				//Create a pointer to read the data
				double* currentdata = new double[size4];
				try
				{
					cfile.Read(currentdata, sizeof(double)*size4);
				}
				catch(CFileException* e)
				{
					TCHAR errormessage[1024];
					e->GetErrorMessage(errormessage, 1024);
					CT2CA ascii(errormessage);
					string errorstring(ascii);
					throw errorstring;
				}
				//Convert the data as vector and append to the vector<vector<vector<double>>>
				data[i][j][k] = PointerToVector(currentdata, size4);
				delete[] currentdata;
			}
		}
	}

	cfile.Close();
	return;
}

//Remove the enter char from a CString
void CBasicFunction::RemoveEnter(string &enterstring)
{
	string::iterator stringiterator;
	for(stringiterator = enterstring.begin(); stringiterator!=enterstring.end(); stringiterator ++)
	{
		if(*stringiterator=='\r')		//If a enter char found, remove it
			enterstring.erase(stringiterator);
	}
	return;
}

//Open a text file, and load the content into a string
string CBasicFunction::LoadFromFile(string filepath)
{
	string filecontent;
	//Open the file
	std::ifstream file(filepath.c_str());
	if(file.fail())
		return string("Failed to open the text");

	//Read the content
	while(!file.eof())
	{
		string currentline;
		std::getline(file, currentline);
		filecontent += currentline;
	}
	file.close();
	return filecontent;
}

unsigned int CBasicFunction::LoadDICOM(vector<unsigned char> datasource, unsigned int & imageheight, unsigned int& imagewidth,
									   vector<unsigned char>& imagedata, string& bitdepth)
{
	unsigned int counter = 0;
	unsigned int datalength = (unsigned int)datasource.size();

	//Read the tag
	if(counter+4>datalength)
		throw string("File format not correct");

	//If encounter image height
	if(datasource[0]==40&&datasource[1]==0&&datasource[2]==16&&datasource[3]==0)	//Image Height
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		unsigned int valuelength = (unsigned int)datasource[counter+2]+(unsigned int)datasource[counter+3]*256;	//Length of value
		counter+=4;
		if(counter+valuelength>datalength)
			throw string("File format not correct");
		if(valuelength==0)
			throw string("Rows not correct");
		imageheight = 0;
		for(unsigned int i=0; i<valuelength; i++)	//Get the value
			imageheight += datasource[counter+i]*(unsigned int)pow((double)256.0, (double)i);
		counter+=valuelength;			//Move the counter forward
	}

	//If encounter image width
	else if(datasource[0]==40&&datasource[1]==0&&datasource[2]==17&&datasource[3]==0)	//Image Height
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		unsigned int valuelength = (unsigned int)datasource[counter+2]+(unsigned int)datasource[counter+3]*256;	//Length of value
		counter+=4;
		if(counter+valuelength>datalength)
			throw string("File format not correct");
		if(valuelength==0)
			throw string("Rows not correct");
		imagewidth = 0;
		for(unsigned int i=0; i<valuelength; i++)	//Get the value
			imagewidth += datasource[counter+i]*(unsigned int)pow((double)256.0, (double)i);
		counter+=valuelength;			//Move the counter forward
	}
	
	//If encounter image depth
	else if(datasource[0]==40&&datasource[1]==0&&datasource[2]==4&&datasource[3]==0)
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		unsigned int valuelength = (unsigned int)datasource[counter+2]+(unsigned int)datasource[counter+3]*256;	//Length of value
		counter+=4;
		if(counter+valuelength>datalength)
			throw string("File format not correct");
		if(valuelength==0)
			throw string("Colour depth not correct");
		bitdepth.resize(valuelength);
		for(unsigned int i=0; i<valuelength; i++)	//Get the intensity depth
			bitdepth[i] = (char)datasource[counter+i];
		counter+=valuelength;			//Move the counter forward
	}

	//If encounter image data
	else if(datasource[0]==224&&datasource[1]==127&&datasource[2]==16&&datasource[3]==0)
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
			unsigned int valuelength = (unsigned int)datasource[counter]+
				(unsigned int)datasource[counter+1]*(unsigned int)pow((double)256, (double)1)+
				(unsigned int)datasource[counter+2]*(unsigned int)pow((double)256, (double)2)+
				(unsigned int)datasource[counter+3]*(unsigned int)pow((double)256, (double)3);		//Value length of the image data
		counter+=4;
		if(counter+valuelength>datalength)
			throw string("File format not correct");
		vector<unsigned char>::iterator startiterator = datasource.begin();
		vector<unsigned char>::iterator enditerator = datasource.begin();
		for(unsigned int i=0; i<counter; i++)
			startiterator ++;
		for(unsigned int i=0; i<counter+valuelength; i++)
			enditerator ++;
		imagedata.resize(0);
		imagedata.insert(imagedata.end(), startiterator, enditerator);
		counter+=valuelength;			//Move the counter forward
	}

	//If encounter sequential data
	else if(datasource[0]==254&&datasource[1]==255&&datasource[2]==0&&datasource[3]==224)
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		if(datasource[counter]==255&&datasource[counter+1]==255&&datasource[counter+2]==255&&datasource[counter+3]==255)	//Undefined length
			counter+=4;			//Move the counter forward
		else			//Defined length
		{
			unsigned int valuelength = (unsigned int)datasource[counter]+
				(unsigned int)datasource[counter+1]*(unsigned int)pow((double)256, (double)1)+
				(unsigned int)datasource[counter+2]*(unsigned int)pow((double)256, (double)2)+
				(unsigned int)datasource[counter+3]*(unsigned int)pow((double)256, (double)3);		//Value length of the image data
			counter+=4;
			if(counter+valuelength>datalength)
				throw string("File format not correct");
			counter+=valuelength;			//Move the counter forward
		}
	}

	//If encounter end of sequential data
	else if(datasource[0]==254&&datasource[1]==255&&datasource[2]==13&&datasource[3]==224)
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		counter+=4;
	}

	else
	{
		counter+=4;
		if(counter+4>datalength)
			throw string("File format not correct");
		if((datasource[counter]=='O'&&datasource[counter+1]=='B')||(datasource[counter]=='O'&&datasource[counter+1]=='W')||
			(datasource[counter]=='U'&&datasource[counter+1]=='N'))
		{
			counter+=4;
			if(counter+4>datalength)
				throw string("File format not correct");
			unsigned int valuelength = (unsigned int)datasource[counter]+
				(unsigned int)datasource[counter+1]*(unsigned int)pow((double)256, (double)1)+
				(unsigned int)datasource[counter+2]*(unsigned int)pow((double)256, (double)2)+
				(unsigned int)datasource[counter+3]*(unsigned int)pow((double)256, (double)3);		//Value length of the image data
			counter+=4;
			counter+=valuelength;		//Move the counter forward
		}
		else if(datasource[counter]=='S'&&datasource[counter+1]=='Q')	//If the VR is SQ
		{
			counter+=4;
			if(counter+8>datalength)
				throw string("File format not correct");
			if(datasource[counter]=='S'&&datasource[counter+1]=='Q')
				counter+=8;
			else
				counter+=4;
		}
		else
		{
			if(counter+4>datalength)
				throw string("File format not correct");
			unsigned int valuelength = (unsigned int)datasource[counter+2]+(unsigned int)datasource[counter+3]*256;
			counter+=4;
			if(counter+valuelength>datalength)
				throw string("File format not correct");
			counter+=valuelength;
		}
	}

	//::AfxMessageBox(douTostr(counter), MB_OK, 0);
	//::AfxMessageBox(douTostr(datalength), MB_OK, 0);

	if(counter==datalength)
		return 1;
	else
	{
		try
		{
			vector<unsigned char>::iterator stringiterator = datasource.begin();
			for(unsigned int i=0; i<counter; i++)
				stringiterator++;
			vector<unsigned char> newdata(stringiterator, datasource.end());
			return CBasicFunction::LoadDICOM(newdata, imageheight, imagewidth, imagedata, bitdepth);
		}
		catch(string exceptionstring)
		{
			throw exceptionstring;
		}
	}
}

//Load a corpus from a file
vector<_corpusunit> CBasicFunction::SetCorpus(string filepath)
{
	vector<_corpusunit> corpus;
	//Load the file
	std::ifstream filestream(filepath.c_str());
	//Get line by and to construct the corpus
	while(!filestream.eof())
	{
		_corpusunit newunit;
		string stringline;
		//Read the line
		std::getline(filestream, stringline);
		//Find the stop between word and frequency
		string::size_type tabposition = stringline.find_last_of('\t');
		if(tabposition==string::npos)
			throw string("File format error");
		newunit.word = string(stringline.c_str(), tabposition);
		newunit.frequency = (unsigned int)atoi(string(&(stringline.c_str()[tabposition+1])).c_str());
		corpus.push_back(newunit);
	}

	if(corpus.size()==0)
		throw string("Selected file is not a corpus file");

	return corpus;
}

//Get the linguistic features of a document
vector<double> CBasicFunction::GetLinguisticFeature(std::string sourcestring, std::vector<_corpusunit> corpus)
{
	vector<double> feature;
	feature.resize(corpus.size());
	for(unsigned int i=0; i<(unsigned int)corpus.size(); i++)
	{
		if(sourcestring.find(corpus[i].word)!=string::npos)		//Find the word in the string
			feature[i] = 1.0;
		else
			feature[i] = 0.0;
	}
	return feature;
}
